package com.jlg.crawl;

import java.net.URL;
import java.sql.Date;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.dom4j.Document;
import org.dom4j.Node;
import org.dom4j.io.SAXReader;
import org.springframework.beans.factory.annotation.Autowired;

import com.jlg.beans.groupon.ApiTemplateBean;
import com.jlg.beans.groupon.CategoryBean;
import com.jlg.beans.groupon.GrouponApiBean;
import com.jlg.beans.groupon.GrouponBean;
import com.jlg.beans.groupon.LocationBean;
import com.jlg.managers.ApiTemplateManager;
import com.jlg.managers.CategoryManager;
import com.jlg.managers.GrouponApiManager;
import com.jlg.managers.GrouponManager;
import com.jlg.managers.LocationManager;
import com.jlg.util.ChineseUtil;
import com.jlg.util.DateUtil;
import com.jlg.util.GoogleMapUtil;
import com.jlg.util.Utils;
import com.julewa.SystemContext;



/**
 * 
 * author: huazhang 
 * since : 2011-3-15
 */
public class GrouponCrawl
{

    public static final String XPATH_PREFIX = "//";
    public static final String CITY_MARKUP = "{city}";
    public static Map<String, String> m_ApiUrl2CityNameMap = new HashMap<String, String>();
    public static Map<String, String> geoLatitudeAndLongitudeMap = new HashMap<String, String>(); // use to cache  Latitude and longitude of geo
    
    public static Map<String, String> hao123CatMap = new HashMap<String, String>();  // 从hao123获取的分类信息是数字ID，需特殊处理，并且类似地均需如此
    
    static
    {
        hao123CatMap.put("1", "餐饮美食");
        hao123CatMap.put("2", "休闲娱乐");
        hao123CatMap.put("3", "美容美发");
        hao123CatMap.put("4", "网上购物");
        hao123CatMap.put("5", "运动健身");
    }
    
    @Autowired ApiTemplateManager apiTemplateManager;
    @Autowired CategoryManager categoryManager;
    @Autowired GrouponApiManager grouponApiManager;
    @Autowired GrouponManager grouponManager;
    @Autowired LocationManager locationManager;
    
    /*************************************************
     * 以此为例演示如何脱离web环境单独启动一个服务或工具
     * 老张同学不要介意哈~~
     * 基本的做法就是在类中按照如下说明添加一个main函数
     * @date 2011-3-18 
     ************************************************/
    public static void main(String[] args)
    {
		/*
		 * 创建一个SystemContext对象以加载系统上下文
		 * 构造函数由一到多个类名构成,他们将被声明为单例
		 * 
		 * 如果类本身已经用@Component等标注为一个服务的话
		 * 则构造函数中可以省去这个类
		 */
       SystemContext ctx = new SystemContext(GrouponCrawl.class);
       /*
        * 从SystemContext中,可以通过如下几个方法获得实例:
        * 1,获得指定类型的实例
        * T t = ctx.getInstanceOf(Class<T>);
        * 2,获得指定类型的所有实例,一般用于父类
        * Map<String, T> m = ctx.getInstancesOf(Class<T>);
        * 3,获得指定id的实例
        * Object o = ctx.getInstanceOf("myResourceId");
        * 4,获得指定类型与id的实例
        * T t = ctx.getInstanceOf(Class<T>, "myResourceId");
        */
       GrouponCrawl crawl = ctx.getInstanceOf(GrouponCrawl.class);
       /*
        * 得到实例后执行相关方法,
        * 这么做的原因是为了方便从main函数接收程序启动参数和其他信息
        */
       crawl.execute();
       /*
        * 当然,以上的代码可缩成一行:
        * new SystemContext(GrouponCrawl.class).getInstanceOf(GrouponCrawl.class).execute();
        * 如果GrouponCrawl被加上了@Component注解的话,这行代码还可以简写为:
        * new SystemContext().getInstanceOf(GrouponCrawl.class).execute();
        */
    }
    
    public void execute()
    {
        try
        {
            SAXReader saxReader = new SAXReader();
            List<GrouponApiBean> groupons = grouponApiManager.getAll();
            Collections.reverse(groupons);      // for test
            for (GrouponApiBean grouponApi : groupons)
            {
                Map<Long, ApiTemplateBean> apiTemplateMap = apiTemplateManager.getAll();
                Set<String> apiUrlSet = new HashSet<String>();
                String apiUrl = grouponApi.getApiurl();
                if (apiUrl.indexOf(CITY_MARKUP) == -1)
                {
                    apiUrlSet.add(apiUrl);
                } else
                {
                    String opencityurl = grouponApi.getOpencityurl();
                    String opencitytag = grouponApi.getOpencitytag(); // 是指定中文城市名称的标签名
                    if (Utils.isEmpty(opencityurl) || Utils.isEmpty(opencitytag))
                    {
                        Utils.ERR("field opencityurl and opencitytag of table t_grouponapi is empty.");
                        continue;
                    }
                    Document document = saxReader.read(new URL(opencityurl));
                    List<Node> nodeList = document.selectNodes(XPATH_PREFIX + opencitytag);
                    for (Node node : nodeList)
                    {
                        String cityName = node.getText();
                        String pinyinCityName = ChineseUtil.chinese2Pinyin(cityName);
                        String copyApiUrl = apiUrl;
                        copyApiUrl = copyApiUrl.replace(CITY_MARKUP,pinyinCityName);
                        m_ApiUrl2CityNameMap.put(copyApiUrl, cityName);
                        apiUrlSet.add(copyApiUrl);
                    }
                }
                for (String aUrl : apiUrlSet)
                {
                    System.out.println("正在抓取" + grouponApi.getName() + " " + aUrl);
                    Document document = null;
                    try
                    {
                        document = saxReader.read(new URL(aUrl));
                    } catch (Exception e)
                    {
                    }
                    if (document == null)
                    {
                        Utils.ERR("failed to crawl url because document is null " + aUrl);
                        continue;
                    }
                    long apitemplateid = grouponApi.getApitemplateid();
                    ApiTemplateBean api = apiTemplateMap.get(apitemplateid);
                    if (api == null)
                    {
                        Utils.ERR("failed to crawl url  because ApiTemplate is null" + aUrl);
                        continue;
                    }
                    String groupbuytag = api.getGroupontag();
                    List<Node> nodeList = document.selectNodes(XPATH_PREFIX + groupbuytag);
                    int size = nodeList.size();
                    for (int i = 1; i <= size; i++)
                    {
                        String xpathPrefix = XPATH_PREFIX + groupbuytag + "[" + i + "]" + XPATH_PREFIX;
                        String location = "";
                        String shop = "";
                        String shopurl = "";
                        String shoplongitude = "";
                        String shoplatitude = "";
                        String shopemail = "";
                        String shoptelephone = "";
                        String shopaddress = "";
                        String shopneighborhood = "";
                        String category = "";
                        String url = "";
                        String weight = "";
                        String title = "";
                        String imageurl = "";
                        String starttime = "";
                        String endtime = "";
                        String value = "";
                        String price = "";
                        String discount = "";
                        String minquantity = "";
                        String maxquantity = "";
                        String bought = "";
                        String detail = "";
                        Node node = null;
                        if (!Utils.isEmpty(api.getLocationtag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getLocationtag());
                            if (node != null)
                            {
                                location = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getShoptag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getShoptag());
                            if (node != null)
                            {
                                shop = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getShopurltag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getShopurltag());
                            if (node != null)
                            {
                                shopurl = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getShoplongitudetag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getShoplongitudetag());
                            if (node != null)
                            {
                                shoplongitude = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getShoplatitudetag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getShoplatitudetag());
                            if (node != null)
                            {
                                shoplatitude = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getShopemailtag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getShopemailtag());
                            if (node != null)
                            {
                                shopemail = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getShoptelephonetag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getShoptelephonetag());
                            if (node != null)
                            {
                                shoptelephone = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getShopaddresstag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getShopaddresstag());
                            if (node != null)
                            {
                                shopaddress = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getShopneighborhoodtag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getShopneighborhoodtag());
                            if (node != null)
                            {
                                shopneighborhood = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getCategorytag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getCategorytag());
                            if (node != null)
                            {
                                category = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getUrltag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getUrltag());
                            if (node != null)
                            {
                                url = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getWeighttag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getWeighttag());
                            if (node != null)
                            {
                                weight = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getTitletag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getTitletag());
                            if (node != null)
                            {
                                title = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getImageurltag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getImageurltag());
                            if (node != null)
                            {
                                imageurl = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getStarttimetag()))
                        {
                            node = document.selectSingleNode(xpathPrefix  + api.getStarttimetag());
                            if (node != null)
                            {
                                starttime = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getEndtimetag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getEndtimetag());
                            if (node != null)
                            {
                                endtime = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getValuetag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getValuetag());
                            if (node != null)
                            {
                                value = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getPricetag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getPricetag());
                            if (node != null)
                            {
                                price = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getDiscounttag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getDiscounttag());
                            if (node != null)
                            {
                                discount = node.getText();
                                discount = discount.replace("折", "");
                            }
                        }
                        if (!Utils.isEmpty(api.getMinquantitytag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getMinquantitytag());
                            if (node != null)
                            {
                                minquantity = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getMaxquantitytag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getMaxquantitytag());
                            if (node != null)
                            {
                                maxquantity = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getBoughttag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getBoughttag());
                            if (node != null)
                            {
                                bought = node.getText();
                            }
                        }
                        if (!Utils.isEmpty(api.getDetailtag()))
                        {
                            node = document.selectSingleNode(xpathPrefix + api.getDetailtag());
                            if (node != null)
                            {
                                detail = node.getText();
                            }
                        }
                        // 处理地理信息(行政区划）
                        long regionLocId = 0;
                        if (Utils.isEmpty(location))
                        {
                            location = m_ApiUrl2CityNameMap.get(aUrl);
                        }
                        if (!Utils.isEmpty(location) && !location.equals("全国"))
                        {
                            long parentId = 0;
                            LocationBean loc = locationManager.findByName(parentId, location,LocationBean.ADMINISTRATIVE_REGINON);
                            if (loc == null)
                            {
                                loc = new LocationBean(); 
                                loc.setParentid(parentId);
                                loc.setType(LocationBean.ADMINISTRATIVE_REGINON);
                                loc.setName(location);
                                loc.setPinyinname(ChineseUtil.chinese2Pinyin(location));
                                loc.setSource(aUrl + "#" + api.getLocationtag());
                                regionLocId = locationManager.save(loc);
                                Utils.info("insert location " + location);
                            } else
                            {
                                regionLocId = loc.getId();
                            }
                        }
                        // 处理地理信息(商圈）
                        long neighborhoodId = 0;
                        if (!Utils.isEmpty(shopneighborhood))
                        {
                            long parentId = regionLocId;
                            LocationBean loc = locationManager.findByName(parentId, shopneighborhood, LocationBean.NEIGHBORHOOD);
                            if (loc == null)
                            {
                                loc = new LocationBean(); 
                                loc.setParentid(parentId);
                                loc.setType(LocationBean.NEIGHBORHOOD);
                                loc.setName(shopneighborhood);
                                loc.setPinyinname(ChineseUtil.chinese2Pinyin(shopneighborhood));
                                loc.setSource(aUrl + "#" + api.getShopneighborhoodtag());
                                neighborhoodId = locationManager.save(loc);
                                Utils.info("insert neighborhood " + shopneighborhood);
                            } else
                            {
                                neighborhoodId = loc.getId();
                            }
                        }
                        // 处理地理信息(团购网站）
                        long grouponSiteLocId = 0;
                        String grouonSiteName = grouponApi.getName();
                        if (!Utils.isEmpty(grouonSiteName))
                        {
                            long parentId = 0;
                            LocationBean loc = locationManager.findByName(parentId, grouonSiteName,LocationBean.GROUPONSHOP);
                            if (loc == null)
                            {
                                loc = new LocationBean(); 
                                loc.setParentid(parentId);
                                loc.setType(LocationBean.GROUPONSHOP);
                                loc.setName(grouonSiteName);
                                loc.setOfficeurl(grouponApi.getSiteurl());
                                loc.setPinyinname(ChineseUtil.chinese2Pinyin(grouonSiteName));
                                loc.setSource(aUrl);
                                grouponSiteLocId = locationManager.save(loc);
                                Utils.info("insert groupon site " + grouonSiteName);
                            } else
                            {
                                grouponSiteLocId = loc.getId();
                            }
                        }
                        // 处理地理信息(商家）
                        long shopLocId = 0;
                        if (!Utils.isEmpty(shop))
                        {
                            long parentId = regionLocId;
                            if (neighborhoodId > 0)
                            {
                                parentId = neighborhoodId;
                            }
                            LocationBean loc = locationManager.findByName(parentId, shop, LocationBean.SHOP);
                            if (loc == null)
                            { 
                                loc = new LocationBean(); 
                                loc.setParentid(parentId);
                                loc.setType(LocationBean.SHOP);
                                loc.setName(shop);
                                loc.setPinyinname(ChineseUtil.chinese2Pinyin(shop));
                                loc.setOfficeurl(shopurl);
                                if (!Utils.isEmpty(shopaddress) && (Utils.isEmpty(shoplongitude) || Utils.isEmpty(shoplatitude)))
                                {
                                    String str = GoogleMapUtil.queryGeographicInfoByGoogleMap(shopaddress);
                                    if (str != null && str.startsWith("200"))
                                    {
                                        String geoLatitudeAndLongitude = geoLatitudeAndLongitudeMap.get(location);
                                        if (Utils.isEmpty(geoLatitudeAndLongitude))
                                        {
                                            String tmp = GoogleMapUtil.queryGeographicInfoByGoogleMap(shopaddress);
                                            if (tmp != null && tmp.startsWith("200"))
                                            {
                                                geoLatitudeAndLongitude = tmp;
                                            }
                                        }
                                        if (!Utils.isEmpty(geoLatitudeAndLongitude) && !str.equals(geoLatitudeAndLongitude))
                                        {
                                            geoLatitudeAndLongitudeMap.put(location, geoLatitudeAndLongitude);
                                            String[] arr = str.split(","); // 200,4,31.2303930,121.4737040
                                            if (arr.length == 4)
                                            {
                                                shoplatitude = arr[2];
                                                shoplongitude = arr[3];
                                            }
                                        }
                                    }
                                }
                                if (!Utils.isEmpty(shoplongitude) && !Utils.isEmpty(shoplatitude))
                                {
                                    loc.setLongitude(Float.parseFloat(shoplongitude));
                                    loc.setLatitude(Float.parseFloat(shoplatitude));
                                }
                                loc.setEmail(shopemail);
                                loc.setTelephone(shoptelephone);
                                loc.setAddress(shopaddress);
                                loc.setSource(aUrl + "#" + api.getShoptag());
                                shopLocId = locationManager.save(loc);
                                Utils.info("insert shop " + shop);
                            } else
                            {
                                shopLocId = loc.getId();
                            }
                        }
                        // 处理分类
                        long catId = 0;
                        if (!Utils.isEmpty(category) && !category.equals("其它"))
                        {
                            // 额外处理分类
                            if (Utils.isNumeric(category))
                            {
                                if (api.getName().toLowerCase().startsWith("hao123"))
                                {
                                    category = hao123CatMap.get(category);
                                }
                               
                            }
                            CategoryBean cat = categoryManager.findByName(category);
                            if (cat == null)
                            {
                                cat = new CategoryBean();
                                cat.setName(category);
                                catId = categoryManager.save(cat);
                                Utils.info("insert category " + category);
                            } else
                            {
                                catId = cat.getId();
                            }
                        }
                        // 处理团购信息
                        if (Utils.isEmpty(url) || grouponSiteLocId == 0 || Utils.isEmpty(title) || Utils.isEmpty(price))
                        {
                            Utils.ERR("groupon info is invalid. " + aUrl);
                        } else
                        {
                            GrouponBean om = grouponManager.findByUrl(url);
                            if (om == null)
                            {
                                om = new GrouponBean();
                                om.setUrl(url);
                                om.setGrouponsiteid(grouponSiteLocId);
                                om.setTitle(title);
                                om.setPrice(Double.parseDouble(price));
                                om.setShopid(shopLocId);
                                om.setLocationid(regionLocId);
                                om.setCategoryid(catId);
                                if (!Utils.isEmpty(weight))
                                {
                                    om.setWeight(Integer.parseInt(weight));
                                }
                                om.setImageurl(imageurl);
                                if (!Utils.isEmpty(starttime))
                                {
                                    Date startTime = null;
                                    if (Utils.isNumeric(starttime))
                                    {
                                        startTime = new Date(Integer.parseInt(starttime));
                                    } else
                                    {
                                        startTime = DateUtil.string2Date(starttime);
                                    }
                                    if (startTime != null)
                                    {
                                        om.setStartTime(startTime.getTime());
                                    }
                                }
                                if (!Utils.isEmpty(endtime))
                                {
                                    Date endTime = null;
                                    if (Utils.isNumeric(endtime))
                                    {
                                        endTime = new Date(Integer.parseInt(endtime));
                                    } else
                                    {
                                        endTime = DateUtil.string2Date(endtime);
                                    }
                                    if (endTime != null)
                                    {
                                        om.setEndTime(endTime.getTime());
                                    }
                                }
                                if (!Utils.isEmpty(value))
                                {
                                    om.setValue(Double.parseDouble(value));
                                }
                                if (!Utils.isEmpty(price))
                                {
                                    om.setPrice(Double.parseDouble(price));
                                }
                                if (!Utils.isEmpty(maxquantity))
                                {
                                    om.setMaxQuantity(Integer.parseInt(maxquantity));
                                }
                                if (!Utils.isEmpty(minquantity))
                                {
                                    om.setMinQuantity(Integer.parseInt(minquantity));
                                }
                                if (!Utils.isEmpty(bought))
                                {
                                    om.setBought(Integer.parseInt(bought));
                                }
                                if (!Utils.isEmpty(discount))
                                {
                                    om.setDiscount(Float.parseFloat(discount));
                                }
                                om.setDetail(detail);
                                long grouponid = grouponManager.save(om);
                                if (grouponid > 0)
                                {
                                    Utils.info("insert groupon info " + grouponid + " " + title);
                                } else
                                {
                                    Utils.ERR("failed to insert grouon info." + title);
                                }
                            }
                        }
                    }
                }
            }
        } catch (Exception e)
        {
            Utils.ERR(e);
        }
    }
}
